{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "req = requests.get('https://wap.zol.com.cn/top/cell_phone/hot.html')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "73269"
     },
     "metadata": {},
     "execution_count": 3
    }
   ],
   "source": [
    "len(req.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "res = re.findall(\n",
    "    '<p class=\"pro-info-name f28\">(.*?)<\\/p>[\\S\\s]*?<span class=\"pro-info-price f24\">(.*?)<\\/span>', req.text\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('mobile.txt', 'w', encoding=\"utf-8\") as f:\n",
    "    for item in res:\n",
    "        f.write(item[0] + ' ' + item[1] + '\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import re\n",
    "\n",
    "class MyCrawler:\n",
    "    def __init__(self, filename):\n",
    "        self.filename = filename\n",
    "    \n",
    "    def download(self, url: str, headers=None, cookies=None):\n",
    "        r = requests.get(url, headers=headers, cookies=cookies)\n",
    "        return r.text\n",
    "\n",
    "    def extract(self, content, pattern):\n",
    "        res = re.findall(pattern, content)\n",
    "        return res\n",
    "    \n",
    "    def save(self, info):\n",
    "        with open(self.filename, 'a', encoding='utf-8') as f:\n",
    "            for item in info:\n",
    "                f.write(''.join(item) + '\\n') \n",
    "\n",
    "    def crawl(self, url: str, pattern: str):\n",
    "        content = self.download(url)\n",
    "        info = self.extract(content, pattern)\n",
    "        self.save(info)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 对zol.com.cn进行测试"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "crawler = MyCrawler('mobile.txt')\n",
    "pat = '<p class=\"pro-info-name f28\">(.*?)<\\/p>[\\S\\s]*?<span class=\"pro-info-price f24\">(.*?)<\\/span>'\n",
    "crawler.crawl('https://wap.zol.com.cn/top/cell_phone/hot.html', pat)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 对哔哩哔哩进行测试"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "b_crawler = MyCrawler('bilibili.txt')\n",
    "url = 'https://www.bilibili.com/ranking?spm_id_from=333.851.b_7072696d61727950616765546162.3'\n",
    "content = b_crawler.download(url)\n",
    "pat = '<a href=\"(.*?)target=\"_blank\"><div class=\"lazy-img cover\"><img alt=\".*?\" src=\"\"></div></a><div class=\"watch-later-trigger w-later\"></div></div><div class=\"info\"><a href=\".*?\" target=\"_blank\" class=\"title\">(.*?)</a><!----><div class=\"detail\"><span class=\"data-box\"><i class=\"b-icon play\"><\\/i>(.*?)<\\/span><span class=\"data-box\"><i class=\"b-icon view\"><\\/i>.*?<\\/span><a target=\"_blank\" href=\".*?\"><span class=\"data-box\"><i class=\"b-icon author\"><\\/i>.*?<\\/span><\\/a><\\/div><div class=\"pts\"><div>(.*?)<\\/div>(综合得分)'\n",
    "res = b_crawler.extract(content, pat)\n",
    "b_crawler.save(res)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 豆瓣读书测试"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "dou_crawler = MyCrawler('douban.txt')\n",
    "\n",
    "url = 'https://book.douban.com/tag/%E7%A5%9E%E7%BB%8F%E7%BD%91%E7%BB%9C'\n",
    "\n",
    "cookies = {\n",
    "    'bid': '^\\\\^KuDYrn0TNJY^\\\\^',\n",
    "    '__yadk_uid': 'VGNAYXf12NFbeeRSW31DHfj7dDSGaRBd',\n",
    "    '__gads': 'ID=8ac9f469e600ef88:T=1591965598:S=ALNI_MbLkAhRH2qcH6cqX7_SzBQCAjAt1Q',\n",
    "    'gr_user_id': '01fb8106-c27d-4da4-9cb9-95ec824b5bc7',\n",
    "    'gr_session_id_22c937bbd8ebd703f2d8e9445f7dfd03': 'a3c6169d-f013-408f-a3a1-1ef1332823ed',\n",
    "    'gr_cs1_a3c6169d-f013-408f-a3a1-1ef1332823ed': 'user_id^%^3A0',\n",
    "    'ap_v': '0,6.0',\n",
    "    '__utma': '30149280.815348947.1591965577.1591965577.1592365074.2',\n",
    "    '__utmc': '30149280',\n",
    "    '__utmz': '30149280.1592365074.2.2.utmcsr=baidu^|utmccn=(organic)^|utmcmd=organic',\n",
    "    '__utmt_douban': '1',\n",
    "    '__utmt': '1',\n",
    "    'gr_session_id_22c937bbd8ebd703f2d8e9445f7dfd03_a3c6169d-f013-408f-a3a1-1ef1332823ed': 'true',\n",
    "    '_vwo_uuid_v2': 'D3B75E3243AB54A93C64754DD415A81DB^|c82ba5a64217bbdbcefa3f4162bd9448',\n",
    "    '_pk_ref.100001.3ac3': '^%^5B^%^22^%^22^%^2C^%^22^%^22^%^2C1592365074^%^2C^%^22https^%^3A^%^2F^%^2Fwww.baidu.com^%^2Flink^%^3Furl^%^3D6JA9-A-UT3kmslX1Ba5uTZxQmqZxrmpfcvtRkcRj6OiBb9_ogLvbEvrRCE5eWucC^%^26wd^%^3D^%^26eqid^%^3Dab924aaf00014e1e000000035ee9900a^%^22^%^5D',\n",
    "    '_pk_ses.100001.3ac3': '*',\n",
    "    '_pk_id.100001.3ac3': '9bf95e667872937d.1591965577.2.1592365079.1591965650.',\n",
    "    '__utmb': '30149280.2.10.1592365074',\n",
    "}\n",
    "\n",
    "headers = {\n",
    "    'Connection': 'keep-alive',\n",
    "    'Cache-Control': 'max-age=0',\n",
    "    'Upgrade-Insecure-Requests': '1',\n",
    "    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4128.3 Safari/537.36',\n",
    "    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',\n",
    "    'Sec-Fetch-Site': 'same-origin',\n",
    "    'Sec-Fetch-Mode': 'navigate',\n",
    "    'Sec-Fetch-User': '?1',\n",
    "    'Sec-Fetch-Dest': 'document',\n",
    "    'Referer': 'https://book.douban.com/',\n",
    "    'Accept-Language': 'zh-CN,zh;q=0.9',\n",
    "}\n",
    "\n",
    "content = dou_crawler.download(url, headers, cookies)\n",
    "dou_crawler.save(content)"
   ]
  }
 ],
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6-final"
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}